Home Erkunden Hilfe
Registrieren Anmelden
J1X
/
huginn
Beobachten 2
Markieren 0
Fork 0

Quellcode durchsuchen
create TwitterSearchAgent

Brian Petro vor 8 Jahren
Ursprung
2646645552
Commit
c7c2f72392
3 geänderte Dateien mit 141 neuen Zeilen und 0 gelöschten Zeilen
Diff-Statistik anzeigen
  1. 97 0
      app/models/agents/twitter_search_agent.rb
  2. 1 0
      spec/data_fixtures/search_tweets.json
  3. 43 0
      spec/models/agents/twitter_search_agent_spec.rb

+ 97 - 0
app/models/agents/twitter_search_agent.rb
Datei anzeigen 

@@ -0,0 +1,97 @@
1 
+module Agents
2 
+  class TwitterSearchAgent < Agent
3 
+    include TwitterConcern
4 
+
5 
+    cannot_receive_events!
6 
+
7 
+    description <<-MD
8 
+      The Twitter Search Agent emits the results of a specified search.
9 
+
10 
+      #{twitter_dependencies_missing if dependencies_missing?}
11 
+
12 
+      To be able to use this Agent you need to authenticate with Twitter in the [Services](/services) section first.
13 
+
14 
+      You must provide the desired `search`.
15 
+
16 
+      Set `result_type` (String) — Specifies what type of search results you would prefer to receive. Options are "mixed", "recent", and "popular". (default: `mixed`)
17 
+
18 
+      Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
19 
+
20 
+      Set `starting_at` to the date/time (eg. `Mon Jun 02 00:38:12 +0000 2014`) you want to start receiving tweets from (default: agent's `created_at`)
21 
+    MD
22 
+
23 
+    event_description <<-MD
24 
+      Events are the raw JSON provided by the [Twitter API](https://dev.twitter.com/rest/reference/get/search/tweets). Should look something like:
25 
+
26 
+          {
27 
+             ... every Tweet field, including ...
28 
+            "text": "something",
29 
+            "user": {
30 
+              "name": "Mr. Someone",
31 
+              "screen_name": "Someone",
32 
+              "location": "Vancouver BC Canada",
33 
+              "description":  "...",
34 
+              "followers_count": 486,
35 
+              "friends_count": 1983,
36 
+              "created_at": "Mon Aug 29 23:38:14 +0000 2011",
37 
+              "time_zone": "Pacific Time (US & Canada)",
38 
+              "statuses_count": 3807,
39 
+              "lang": "en"
40 
+            },
41 
+            "retweet_count": 0,
42 
+            "entities": ...
43 
+            "lang": "en"
44 
+          }
45 
+    MD
46 
+
47 
+    default_schedule "every_1h"
48 
+
49 
+    def working?
50 
+      event_created_within?(interpolated['expected_update_period_in_days']) && !recent_error_logs?
51 
+    end
52 
+
53 
+    def default_options
54 
+      {
55 
+        'search' => 'freebandnames',
56 
+        'expected_update_period_in_days' => '2'
57 
+      }
58 
+    end
59 
+
60 
+    def validate_options
61 
+      errors.add(:base, "search is required") unless options['search'].present?
62 
+      errors.add(:base, "expected_update_period_in_days is required") unless options['expected_update_period_in_days'].present?
63 
+
64 
+      if options[:starting_at].present?
65 
+        Time.parse(options[:starting_at]) rescue errors.add(:base, "Error parsing starting_at")
66 
+      end
67 
+    end
68 
+
69 
+    def starting_at
70 
+      if interpolated[:starting_at].present?
71 
+        Time.parse(interpolated[:starting_at]) rescue created_at
72 
+      else
73 
+        created_at
74 
+      end
75 
+    end
76 
+
77 
+    def check
78 
+      since_id = memory['since_id'] || nil
79 
+      opts = {:include_entities => true}
80 
+      opts.merge! result_type: interpolated[:result_type] if interpolated[:result_type].present?
81 
+      opts.merge! :since_id => since_id unless since_id.nil?
82 
+
83 
+      # http://www.rubydoc.info/gems/twitter/Twitter/REST/Search
84 
+      tweets = twitter.search(interpolated['search'], opts).take(100)
85 
+
86 
+      tweets.each do |tweet|
87 
+        if tweet.created_at >= starting_at
88 
+          memory['since_id'] = tweet.id if !memory['since_id'] || (tweet.id > memory['since_id'])
89 
+
90 
+          create_event :payload => tweet.attrs
91 
+        end
92 
+      end
93 
+
94 
+      save!
95 
+    end
96 
+  end
97 
+end

+ 1 - 0
spec/data_fixtures/search_tweets.json
Datei anzeigen 

@@ -0,0 +1 @@
1 
+{"statuses":[{"metadata":{"result_type":"recent","iso_language_code":"en"},"created_at":"Fri Dec 20 16:52:25 +0000 2013","id":414075829182676992,"id_str":"414075829182676992","text":"@Just_Reboot #FreeBandNames mono surround","source":"\u003ca href=\"http:\/\/www.myplume.com\/\" rel=\"nofollow\"\u003ePlume\u00a0for\u00a0Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":414069174856843264,"in_reply_to_status_id_str":"414069174856843264","in_reply_to_user_id":29296581,"in_reply_to_user_id_str":"29296581","in_reply_to_screen_name":"Just_Reboot","user":{"id":546527520,"id_str":"546527520","name":"Phil Empanada","screen_name":"ItsFuckinOhSo","location":"By cacti and sand","description":"Insert cheesy warnings about this account","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":257,"friends_count":347,"listed_count":10,"created_at":"Fri Apr 06 02:15:16 +0000 2012","favourites_count":345,"utc_offset":-25200,"time_zone":"Arizona","geo_enabled":false,"verified":false,"statuses_count":21765,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/414512901680930816\/AcmN-ByT_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/414512901680930816\/AcmN-ByT_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/546527520\/1371244104","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"FreeBandNames","indices":[13,27]}],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"Just_Reboot","name":"Reboot","id":29296581,"id_str":"29296581","indices":[0,12]}]},"favorited":false,"retweeted":false,"lang":"en"},{"metadata":{"result_type":"recent","iso_language_code":"en"},"created_at":"Fri Dec 20 16:43:32 +0000 2013","id":414073595372265472,"id_str":"414073595372265472","text":"RT @Just_Reboot: The Dick Tonsils #FreeBandNames\u00a0","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":109670150,"id_str":"109670150","name":"Derek Rodriguez","screen_name":"drod2169","location":"Tampa, Florida","description":"Web Developer, College Student, Workout Addict, Taco Fiend. #FBGT","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":3119,"friends_count":714,"listed_count":142,"created_at":"Fri Jan 29 21:29:16 +0000 2010","favourites_count":3728,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":46991,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/3347464379\/1046c92ab1834f1a3c1692ea585a1d69_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/3347464379\/1046c92ab1834f1a3c1692ea585a1d69_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/109670150\/1368729387","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"metadata":{"result_type":"recent","iso_language_code":"en"},"created_at":"Fri Dec 20 16:34:40 +0000 2013","id":414071361066532864,"id_str":"414071361066532864","text":"The Dick Tonsils #FreeBandNames\u00a0","source":"\u003ca href=\"http:\/\/www.myplume.com\/\" rel=\"nofollow\"\u003ePlume\u00a0for\u00a0Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":29296581,"id_str":"29296581","name":"Reboot","screen_name":"Just_Reboot","location":"NC","description":"G+ http:\/\/t.co\/MyXmhCAtnD\r\n#TeamKang PR, Attempted Writer, IT Tech, Social Media Berserker and Entertainer, Adjectives, Assertions, Disclaimers","url":null,"entities":{"description":{"urls":[{"url":"http:\/\/t.co\/MyXmhCAtnD","expanded_url":"http:\/\/goo.gl\/giVTc","display_url":"goo.gl\/giVTc","indices":[3,25]}]}},"protected":false,"followers_count":2244,"friends_count":435,"listed_count":60,"created_at":"Mon Apr 06 21:21:27 +0000 2009","favourites_count":79,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":27703,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/871578268\/5db26751bf732750f5ca1ed4f9f59309.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/871578268\/5db26751bf732750f5ca1ed4f9f59309.png","profile_background_tile":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/378800000480531175\/93e8b5cb95a635ebb81302f5b6044a76_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/378800000480531175\/93e8b5cb95a635ebb81302f5b6044a76_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/29296581\/1368898490","profile_link_color":"009999","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"EFEFEF","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":1,"favorite_count":1,"entities":{"hashtags":[{"text":"FreeBandNames","indices":[17,31]}],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"lang":"en"},"retweet_count":1,"favorite_count":0,"entities":{"hashtags":[{"text":"FreeBandNames","indices":[34,48]}],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"Just_Reboot","name":"Reboot","id":29296581,"id_str":"29296581","indices":[3,15]}]},"favorited":false,"retweeted":false,"lang":"en"},{"metadata":{"result_type":"recent","iso_language_code":"en"},"created_at":"Fri Dec 20 16:34:40 +0000 2013","id":414071361066532864,"id_str":"414071361066532864","text":"The Dick Tonsils #FreeBandNames\u00a0","source":"\u003ca href=\"http:\/\/www.myplume.com\/\" rel=\"nofollow\"\u003ePlume\u00a0for\u00a0Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":29296581,"id_str":"29296581","name":"Reboot","screen_name":"Just_Reboot","location":"NC","description":"G+ http:\/\/t.co\/MyXmhCAtnD\r\n#TeamKang PR, Attempted Writer, IT Tech, Social Media Berserker and Entertainer, Adjectives, Assertions, Disclaimers","url":null,"entities":{"description":{"urls":[{"url":"http:\/\/t.co\/MyXmhCAtnD","expanded_url":"http:\/\/goo.gl\/giVTc","display_url":"goo.gl\/giVTc","indices":[3,25]}]}},"protected":false,"followers_count":2244,"friends_count":435,"listed_count":60,"created_at":"Mon Apr 06 21:21:27 +0000 2009","favourites_count":79,"utc_offset":-18000,"time_zone":"Eastern Time (US & Canada)","geo_enabled":false,"verified":false,"statuses_count":27703,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/871578268\/5db26751bf732750f5ca1ed4f9f59309.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/871578268\/5db26751bf732750f5ca1ed4f9f59309.png","profile_background_tile":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/378800000480531175\/93e8b5cb95a635ebb81302f5b6044a76_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/378800000480531175\/93e8b5cb95a635ebb81302f5b6044a76_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/29296581\/1368898490","profile_link_color":"009999","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"EFEFEF","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":1,"favorite_count":1,"entities":{"hashtags":[{"text":"FreeBandNames","indices":[17,31]}],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"lang":"en"}],"search_metadata":{"completed_in":0.012,"max_id":414075829182676992,"max_id_str":"414075829182676992","next_results":"?max_id=414071361066532863&q=%23freebandnames&count=3&include_entities=1","query":"%23freebandnames","refresh_url":"?since_id=414075829182676992&q=%23freebandnames&include_entities=1","count":3,"since_id":0,"since_id_str":"0"}}

+ 43 - 0
spec/models/agents/twitter_search_agent_spec.rb
Datei anzeigen 

@@ -0,0 +1,43 @@
1 
+require 'spec_helper'
2 
+
3 
+describe Agents::TwitterSearchAgent do
4 
+  before do
5 
+    # intercept the twitter API request
6 
+    stub_request(:any, /freebandnames/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/search_tweets.json")), :status => 200)
7 
+
8 
+    @opts = {
9 
+      :search => "freebandnames",
10 
+      :expected_update_period_in_days => "2",
11 
+      :starting_at => "Jan 01 00:00:01 +0000 2000",
12 
+      :consumer_key => "---",
13 
+      :consumer_secret => "---",
14 
+      :oauth_token => "---",
15 
+      :oauth_token_secret => "---"
16 
+    }
17 
+
18 
+    @checker = Agents::TwitterSearchAgent.new(:name => "search freebandnames", :options => @opts)
19 
+    @checker.service = services(:generic)
20 
+    @checker.user = users(:bob)
21 
+    @checker.save!
22 
+  end
23 
+
24 
+  describe "#check" do
25 
+    it "should check for changes" do
26 
+      expect { @checker.check }.to change { Event.count }.by(100)
27 
+    end
28 
+  end
29 
+
30 
+  describe "#check with starting_at=future date" do
31 
+    it "should check for changes starting_at a future date, thus not find any" do
32 
+      opts = @opts.merge({ :starting_at => "Jan 01 00:00:01 +0000 2999" })
33 
+
34 
+      checker = Agents::TwitterSearchAgent.new(:name => "searching freebandnames", :options => opts)
35 
+      checker.service = services(:generic)
36 
+      checker.user = users(:bob)
37 
+      checker.save!
38 
+
39 
+      expect { checker.check }.to change { Event.count }.by(0)
40 
+    end
41 
+  end
42 
+
43 
+end